
#SCRIPT CU CODUL COMPLET-cele 3 modele antrenate


!pip install warnings
!pip install numpy

ERROR: Could not find a version that satisfies the requirement warnings (from versions: none)
ERROR: No matching distribution found for warnings

Requirement already satisfied: numpy in c:\users\gabrielabosca\anaconda3\lib\site-packages (1.24.3)


import warnings 
warnings.filterwarnings("ignore", category=DeprecationWarning)
warnings.filterwarnings("ignore", category=UserWarning)
warnings.filterwarnings("ignore", category=FutureWarning)

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
%matplotlib inline
#comanda specifica jupyter notebook ce indica ca iesirile de la ploturi is afisate direct in notebook la celula unde sunt facute
import seaborn as sns
sns.set()
from PIL import Image
import math
from matplotlib.colors import ListedColormap
from matplotlib.lines import Line2D

import torch
import torch.nn as nn
import torch.optim as optim
from torch.optim.lr_scheduler import ReduceLROnPlateau, StepLR, CyclicLR
import torchvision
from torchvision import datasets, models, transforms
from torch.utils.data import Dataset, DataLoader
import torch.nn.functional as F

from scipy.special import expit
from sklearn.metrics import confusion_matrix, f1_score, precision_score, recall_score, accuracy_score
from sklearn.model_selection import train_test_split, StratifiedKFold
from sklearn.utils.class_weight import compute_class_weight

from glob import glob
from skimage.io import imread
from os import listdir

import time
import copy
from tqdm import tqdm_notebook as tqdm


#SETTING

run_training_relu512 = False
run_training_relu1024 = False
run_training_softmax = False
#retrain = False
find_learning_rate = True


#HOW MANY PATIENTS DO WE HAVE?

base_path = "D:/input/breast-histopathology-images/IDC_regular_ps50_idx5/"
folder = listdir(base_path)
len(folder)

279


#HOW THE FILES LOOK LIKE?

folder[0:15]

['10253',
 '10254',
 '10255',
 '10256',
 '10257',
 '10258',
 '10259',
 '10260',
 '10261',
 '10262',
 '10264',
 '10268',
 '10269',
 '10272',
 '10273']


#HOW MANY PATCHES DO WE HAVE IN TOTAL? images of cancerous and healthy tissue

total_images = 0
for n in range(len(folder)): #trecem prin fiecare fisier(pacient) pe rand
    patient_id = folder[n]
    for c in [0, 1]:
        patient_path = base_path + patient_id
        class_path = patient_path + "/" + str(c) + "/" #path ul spre directorul pentru clasa curenta 0/1 din directorul unui pacient 
        subfiles = listdir(class_path) #preia lista de imagini din directorul curent
        total_images += len(subfiles) #aduna nr de imagini din clasa curenta la nr total de imagini


total_images #ne va arata cate imagini sunt in total in baza noastra de date

277524


#STORING THE image_path, patient_id and the target

#facem un dataframe gol cu nr de linii(total_images) si 3 coloane; fiecare rand corespunde unei imagini din baza de date
data = pd.DataFrame(index=np.arange(0, total_images), columns=["patient_id", "path", "target"])

k = 0
for n in range(len(folder)): #trecem prin fiecare pacient
    patient_id = folder[n] #ia id ul pacientului
    patient_path = base_path + patient_id #se face path ul spre fisierul ce contine imaginile pentru pacientul curent
    for c in [0, 1]: #trecem prin fiecare caz bolnav/sanatos
        class_path = patient_path + "/" + str(c) + "/" #se face path ul spre directorul cu imagini pt clasa curenta 0 sau 1
        subfiles = listdir(class_path) #recuperam lista de imagini din directorul clasa curent
        for m in range(len(subfiles)): #trecem prin fiecare imagine pentru a popula dataframe ul
            image_path = subfiles[m] #ia numele fisierului imaginii curente
            data.iloc[k]["path"] = class_path + image_path #adauga path ul in dataframe
            data.iloc[k]["target"] = c #adauga clasa in DF 0/1
            data.iloc[k]["patient_id"] = patient_id # adauga id ul pacientului cu poza curenta in DF
            k += 1 #incrementam k cu 1 ca sa trecem la urmatorul rand
            
data.head() #afiseaza primele randuri din DF ca sa inspectam informatia stocata


print(data["target"].unique()) #asta am adaugat ca sa ma asigur ca programul vede atat fisierele 0 cat si 1

[0 1]


data.shape #ne va arata cate randuri si coloane avem in DF, practic verificam daca am lucrat corect

(277524, 3)


cancer_perc = data.groupby("patient_id").target.value_counts()/ data.groupby("patient_id").target.size()
cancer_perc = cancer_perc.unstack()

fig, ax = plt.subplots(1,3,figsize=(20,5))
plot1 = "#FFEB85"
plot2 = "#FF4A86"
healthy = "#92FF33"
cancerous = "#FF5E5E"

# Primul plot
sns.distplot(data.groupby("patient_id").size(), ax=ax[0], color=plot1, kde=False, bins=30)
ax[0].set_xlabel("Number of patches")
ax[0].set_ylabel("Frequency")
ax[0].set_title("How many patches do we have per patient?")

# Al doilea plot
sns.distplot(cancer_perc.loc[:, 1] * 100, ax=ax[1], color=plot2, kde=False, bins=30)
ax[1].set_ylabel("Frequency")
ax[1].set_xlabel("% of patches with IDC")
ax[1].set_title("How much percentage of an image is covered by IDC?")


sns.distplot(data[data['target'] == 0]['target'], kde=False, color=healthy, ax=ax[2], bins=1, label='No IDC')
sns.distplot(data[data['target'] == 1]['target'], kde=False, color=cancerous, ax=ax[2], bins=1, label='IDC Present')
ax[2].set_xlabel("No (0) versus Yes (1)")
ax[2].set_ylabel("Frequency")
ax[2].set_title("Distribution of IDC Presence")
ax[2].set_xlim(-0.5, 1.5)
ax[2].set_xticks([0, 1])
ax[2].xaxis.set_minor_locator(plt.FixedLocator([0.5]))


#LOOKING AT HEALTHY AND CANCER PATCHES

data.target = data.target.astype(int) #converteste 'target' din DF 'data' in integer


pos_selection = np.random.choice(data[data.target==1].index.values, size=50, replace=False) #selecteaza random 50 de indici dn randurile unde 'target' = 1 adica patch uri canceroase
neg_selection = np.random.choice(data[data.target==0].index.values, size=50, replace=False) #acelasi lucru doar ca pt patch uri sanatoase
#replace=False asigura ca fiecare index selectat este unic si ca nu e folosit de mai multe ori, deci 50 de cazuri diferite pt fiecare situatie


#CANCER PATCHES

#vom afisa o figura cu subploturi pe 5 randuri si 10 coloane
fig, ax = plt.subplots(5,10,figsize=(20,10))

for n in range(5): #mergem pe fiecare rand
    for m in range(10): #mergem pe fiecare coloana
        idx = pos_selection[m +10*n] #calculeaza indexul imaginii care va fi afisata pe baza randului si a coloanei curente folosind indexul preselecat din 'pos_selection' ce l am facut mai sus
        image = imread(data.loc[idx, "path"]) #citeste path ul imaginii din DataFrame folosind indexul calculat 'idx'
        ax[n,m].imshow(image) #afiseaza imagiea pe subplotul curent la pozitia (n,m)
        ax[n,m].grid(False) #ascunde grid lines pt fiecare subplot


#HEALTHY PATCHES 

#vom face acelasi lucru ca mai sus doar ca pentru patch urile sanatoase
fig, ax = plt.subplots(5,10,figsize=(20,10))

for n in range(5):
    for m in range(10):
        idx = neg_selection[m + 10*n]
        image = imread(data.loc[idx, "path"])
        ax[n,m].imshow(image)
        ax[n,m].grid(False)


#Visualising the breast tissue

def extract_coords(df): #ia un DataFrame drept input ca df care ar trebui sa contina o coloana numita 'path' care sa reprezinte path ul pt fisiere
    coord = df.path.str.rsplit("_", n=4, expand=True) #o sa separe fiecare path prin '_'limiteaza separareea la 4 ca sa ia coordonatele si sa elimine partile nefolositoare din numele fisierului
    coord = coord.drop([0, 1, 4], axis=1) #elimina partile nefolositoare 
    coord = coord.rename({2: "x", 3: "y"}, axis=1) #redenumin coloanele care raman cu 'x' si 'y'
    coord.loc[:, "x"] = coord.loc[:,"x"].str.replace("x", "", case=False).astype(int) #elimina caracterele 'x'din coloanele coordonate si le converteste in integers
    coord.loc[:, "y"] = coord.loc[:,"y"].str.replace("y", "", case=False).astype(int) #la fel si pt 'y'
    df.loc[:, "x"] = coord.x.values #adauga 'x' estras din coloanele coordnate la DataFrame original 'df'
    df.loc[:, "y"] = coord.y.values # la fel si pt 'y'
    return df

def get_cancer_dataframe(patient_id, cancer_id): #se construieste un DataFrame care contine info despre exemplarele de tesut pt un anumit pacient si tip de cancer '0' sau '1'
    path = base_path + patient_id + "/" + cancer_id #se construiest path ul catre directorul care contine exemplarele de tesut in functie de id ul pacientului si de cancr id ('0' sau '1')
    files = listdir(path) #listeaza fisierele din directorul specificat folosind listdir() din modulul 'os'
    dataframe = pd.DataFrame(files, columns=["filename"])  #se face un DataFrame numit 'dataframe' car o sa aiba o singura coloana ce va contine numele fisierelor
    path_names = path + "/" + dataframe["filename"].values # se face path ul intreg catre fisier prin concatenarea path ului la dirctor cu numele fisierelor
    dataframe = dataframe["filename"].str.rsplit("_", n=4, expand=True) #se impart filenams urile ca sa se extraga coordonatle(x,y)
    dataframe.loc[:, "target"] = int(cancer_id) #se asigneaza valoarea target adica cancer_id (0 sau 1) fiecarui exemplar
    dataframe.loc[:, "path"] = path_names
    dataframe = dataframe.drop([0, 1, 4], axis=1) #se elimina partile nefolositoare 
    dataframe = dataframe.rename({2: "x", 3: "y"}, axis=1) #se redenumesc coloanele 'x' si 'y'
    dataframe.loc[:, "x"] = dataframe.loc[:,"x"].str.replace("x", "", case=False).astype(int)
    dataframe.loc[:, "y"] = dataframe.loc[:,"y"].str.replace("y", "", case=False).astype(int)
    return dataframe

def get_patient_dataframe(patient_id): #aici se face un alt DtaFrame ce contine info despre exemplarele de tesut ale unui pacient specific prin concatenarea DataFrame uui de la 'get_cancer_dataframe' pentru cazurile 0 si 1 
    df_0 = get_cancer_dataframe(patient_id, "0")
    df_1 = get_cancer_dataframe(patient_id, "1")
    #se vor gestiona mai multe cazuri
    #daca nu avem in niciun fisier nimic, se va returna un df gol, daca doar dintr un fisier lispeste informatia, atunci se va returta DataFrame ul pt cazul disponibil (0 sau 1)
    if df_0 is None and df_1 is None: 
        return pd.DataFrame() 
    elif df_0.empty and df_1.empty:
        return pd.DataFrame() 
    elif not df_0.empty and not df_1.empty:
        patient_df = pd.concat([df_0, df_1], ignore_index=True)
        return patient_df
    elif not df_0.empty:
        return df_0
    elif not df_1.empty:
        return df_1


example = get_patient_dataframe(data.patient_id.values[0])
example.head()


# BINARY TARGET VISUALIZATION PER TISSUE SLICE
fig, ax = plt.subplots(5, 3, figsize=(20, 27))

patient_ids = data.patient_id.unique() #preia id uri unice ale pacientilor din DF 

# Define a custom colormap
custom_cmap = ListedColormap(['green', 'red'])

# Define the labels for the legend
legend_labels = ['no IDC', 'IDC']

for n in range(5):
    for m in range(3):
        patient_id = patient_ids[m + 3 * n]
        example_df = get_patient_dataframe(patient_id)
        
        # Scatter plot
        scatter = ax[n, m].scatter(example_df.x.values, example_df.y.values, c=example_df.target.values, cmap=custom_cmap, s=20)
        
        # Add legend manually
        legend_elements = [Line2D([0], [0], marker='o', color='w', markersize=10, markerfacecolor='green', label='Target 0'),
                           Line2D([0], [0], marker='o', color='w', markersize=10, markerfacecolor='red', label='Target 1')]
        ax[n, m].legend(handles=legend_elements, labels=legend_labels, loc='upper right')
        
        ax[n, m].set_title("patient " + patient_id)
        ax[n, m].set_xlabel("y coord")
        ax[n, m].set_ylabel("x coord")

plt.tight_layout()
plt.show()


#VISUALISING THE BREAST TISSUE IMAGES
#aici practic o sa vizualizam tesutul 

def visualise_breast_tissue(patient_id, pred_df=None): #se preia id ul pacientului de la care se va vizualiza tesutul
                                       #pred_df e optional, e un DataFrame ce contine predictii pt exemplarele de tesut ale unui pacient dat 
                                       #daca e dat, functia va genera vizualizari suplimentare cu măști de probabilitate bazate pe predicții
    example_df = get_patient_dataframe(patient_id) #incepem prin a obtine un df 'example_df' ce contine info destpre tesutul unui pacient dat folosind 'get_patient_dataframe'
    max_point = [example_df.y.max()-1, example_df.x.max()-1] #se calculeaza max x si y ca sa se determine dimensiunea gridului si sirurile de masti
    #se initializeaza grid si mask cu background alb
    grid = 255*np.ones(shape = (max_point[0] + 50, max_point[1] +50, 3)).astype(np.uint8)
    mask = 255*np.ones(shape = (max_point[0] + 50, max_point[1] +50, 3)).astype(np.uint8)
    if pred_df is not None: #daca au fost procurate predictii, se selecteaza predictiile corespunzatoare id ului pacientului dat
        patient_df = pred_df[pred_df.patient_id == patient_id].copy()
    mask_proba = np.zeros(shape=(max_point[0] + 50, max_point[1] + 50, 1)).astype(float)
    
    broken_patches = []
    for n in range(len(example_df)): #se trece prin fiecare exemplar de tesut din 'example_df'
        try:
            image = imread(example_df.path.values[n]) #se citeste imaginea
            
            target = example_df.target.values[n] #se extrage targetul 0 sau 1
            #aici se calculeaza coordonatele si simensiunile ca sa se plaseze exemplarul de tesut in grid
            x_coord = int(example_df.x.values[n])
            y_coord = int(example_df.y.values[n])
            x_start = x_coord - 1
            y_start = y_coord - 1
            x_end = x_start + 50
            y_end = y_start + 50
            
            grid[y_start:y_end, x_start:x_end] = image #se suprapune exemplarul de tesut pe grid
            if target == 1: #daca exemplarul de tesut e canceros (target=1) se actualizeaza masca ca sa evidentieze zona bolnava
                mask[y_start:y_end, x_start:x_end, 0] = 250
                mask[y_start:y_end, x_start:x_end, 1] = 0
                mask[y_start:y_end, x_start:x_end, 2] = 0
            if pred_df is not None: #daca avem si predictii se extrage probabilitatea prezisa din 'pred_df' si se actualizeaza 'mask_proba'
                
                proba = patient_df[
                    (patient_df.x==x_coord) & (patient_df.y==y_coord)].proba
                mask_proba[y_start:y_end, x_start:x_end, 0] = proba
                
        except ValueError:
            broken_patches.append(example_df.path.values[n])
    #in final vom returna    
    return grid, mask, broken_patches, mask_proba
    #grid = locul unde imaginea cu tesut va fi plasata 
    #mask = va evidentia zona canceroasa
    #broken_patches = o lista ce stocheaza path ul pt orice exemplar de tesut ce nu a putut fi prelucrat corespunzator
    #mask_proba = un array ce reprezinta masca de probabilitate a regiunii canceroase. e goala initial si se va umple cu probabilitati prezise daca  'pred_df' e dat


#luam un exemplu sa verificam
example = "13616"
grid, mask, broken_patches,_ = visualise_breast_tissue(example)
#am apelat functia 'visualise_breast_tissue' ca sa se obtina 'grid' si 'mask' ce reprezinta imaginea tesutului si regiunile canceroase 
fig, ax = plt.subplots(1,2,figsize=(20,10))
ax[0].imshow(grid, alpha=0.9) #pe prima fig se afiseaza imaginea cu tesut cu transparenta putina/mica care e data de parametrul 'alpha'
ax[1].imshow(mask, alpha=0.8) #pe a doua fig se afiseaza zona bolnava suprapusa pe imaginea cu tesut cu niste transparenta
ax[1].imshow(grid, alpha=0.7) #de asemenea tot in a doua fig se afiseaza imaginea de tesut
ax[0].grid(False)
ax[1].grid(False)
for m in range(2):
    ax[m].set_xlabel("y-coord")
    ax[m].set_ylabel("y-coord")
ax[0].set_title("Breast tissue slice of patient: " + example)
ax[1].set_title("Cancer tissue colored red \n of patient: " + example);


broken_patches #verificam daca sunt exemplare de tesut ce nu au putut fi procesate

[]


#Setting up the machine learning workflow 

BATCH_SIZE = 32 #nr de mostre ce trebuie procesate la fiecare iteratie de training sau lot
NUM_CLASSES = 2 #nr de clase din classification task

OUTPUT_PATH = "D:/input/breastcancermodel/" #path ul unde o sa fie salvate fisiere cum ar fi trained models si evaluation results 
MODEL_PATH = "D:/input/breastcancermodel/" #path ul unde o sa fie salvate trained models (modelele antrenate)
LOSSES_PATH = "D:/input/breastcancermodel/" #path ul unde informatia despre pierderi sau metricele o sa fie salvate in timpul antrenarii 
ACC_PATH = "D:/input/breastcancermodel/" #path ul unde informatia despre acuratete sau metricele o sa fie salvate in timpul antrenarii


#VALIDATION STRATEGY

data.head() #se vor afisa cateva randuri din DF 'data' ca sa inspectam si sa verificam ca s a incarcat corect
data.loc[:, "target"] = data.target.astype(str) #se converteste tipul de date din coloana 'target' in string
data.info() #se va printa un rezumat concis. ofera info utile despre structura si compozitia DF ului
#conversia in str se face ca sa asigure consistenta in tipurile de date, mai ales daca coloana 'target' contine variabile categoriale sau daca vrem sa le tratam ca a atare

<class 'pandas.core.frame.DataFrame'>
Index: 277524 entries, 0 to 277523
Data columns (total 3 columns):
 #   Column      Non-Null Count   Dtype 
---  ------      --------------   ----- 
 0   patient_id  277524 non-null  object
 1   path        277524 non-null  object
 2   target      277524 non-null  object
dtypes: object(3)
memory usage: 13.5+ MB


#se vor separa id uri unice ale pacientilor in seturi pt antrenare, testare si dezvoltare
patients = data.patient_id.unique() #ia id uri din DFul 'data', coloana 'patient_id' si le asigneaza variabilei 'patients'

train_ids, sub_test_ids = train_test_split(patients, #se va imparti setul 'patients' in 2 
                                          test_size=0.3,#30% din date vor fi folosite pt test deci 70% pt antrenare
                                          random_state=0) #e 0 ca sa asigure reproductibilitatea
#mai apoi vom imparti setul 'sub_test_ids' pe care l am obt mai sus in 2 seturi
test_ids, dev_ids = train_test_split(sub_test_ids,
                                     test_size=0.5, #50% din datele ce raman (dupa prima impartire) o sa fie folosite pt test (test_ids) si 50% pt dezvoltare(dev_ids)
                                     random_state=0)


print(len(train_ids)/patients.shape[0]*100, #calculam procentajul de pacienti din setul de training
     len(dev_ids)/patients.shape[0]*100, #acelasi lucru doar ca in setul de dezvoltare
     len(test_ids)/patients.shape[0]*100) #acelasi lucru doar ca in testare

69.89247311827957 15.053763440860216 15.053763440860216


#printam nr de id uri de pacienti alocat fiecarui set
print(len(train_ids), len(dev_ids), len(test_ids))

195 42 42


#se va imparti DF ul original 'data' in 3 DFuri in fct de id urile pacientilor
train_df = data.loc[data.patient_id.isin(train_ids), :].copy()
test_df = data.loc[data.patient_id.isin(test_ids), :].copy()
dev_df = data.loc[data.patient_id.isin(dev_ids), :].copy()
#se aplica functia 'extract_coords()' fiecarui DF creat pt a extrage coordonatele din path urile fisierelor si sa le stocheze in coloanele 'x' si 'y'
train_df = extract_coords(train_df)
test_df = extract_coords(test_df)
dev_df = extract_coords(dev_df)


#printam coloanele din fiecare DF ca sa vedem daca am reusit sa extragem
print(train_df.columns)
print(dev_df.columns)
print(test_df.columns)

Index(['patient_id', 'path', 'target', 'x', 'y'], dtype='object')
Index(['patient_id', 'path', 'target', 'x', 'y'], dtype='object')
Index(['patient_id', 'path', 'target', 'x', 'y'], dtype='object')


#TARGET DISTRIBUTION
# Definirea paletelor de culori
pink = ["#D83465", "#FF80AE"] 
purple = ["#B356E2", "#B883DB"]
yellow = ["#FFF836", "#FFFACD"]

# Crearea figurii și a axelor pentru ploturi
fig, axs = plt.subplots(1, 3, figsize=(18, 6))

# TARGET DISTRIBUTION FOR 'train_df' SET
sns.countplot(x='target', data=train_df, palette=purple, ax=axs[0])
axs[0].set_title("TARGET DISTRIBUTION FOR 'train_df' SET")
axs[0].set_xlabel("Target ('0' or '1')")
axs[0].set_ylabel("Frequency")
axs[0].set_xticks([0, 1])
axs[0].set_xticklabels(["No IDC(0)", "IDC Present(1)"])

# TARGET DISTRIBUTION FOR 'test_df' SET
sns.countplot(x='target', data=test_df, palette=pink, ax=axs[1])
axs[1].set_title("TARGET DISTRIBUTION FOR 'test_df' SET")
axs[1].set_xlabel("Target ('0' or '1')")
axs[1].set_ylabel("Frequency")
axs[1].set_xticks([0, 1])
axs[1].set_xticklabels(["No IDC(0)", "IDC Present(1)"])

# TARGET DISTRIBUTION FOR 'dev_df' SET
sns.countplot(x='target', data=dev_df, palette=yellow, ax=axs[2])
axs[2].set_title("TARGET DISTRIBUTION FOR 'dev_df' SET")
axs[2].set_xlabel("Target ('0' or '1')")
axs[2].set_ylabel("Frequency")
axs[2].set_xticks([0, 1])
axs[2].set_xticklabels(["No IDC(0)", "IDC Present(1)"])

plt.tight_layout()
plt.show()


class BreastCancerDataset(Dataset): #facem o clasa pt incarcarea si procesarea datelor
    #definm 3 metode
    def __init__(self, df, transform=None): #df reprezinta DataFrame ce contine info despre setul de date, transform rep trasformarile imaginilor ce vor fi aplicate fiecarei imagini din set, daca nu sunt transformari e by defauld None
        self.states = df #self.states e un atribut ce stocheaza DataFrame ce contine info despre setul de date
        self.transform=transform #self.transform e un atribut ce stocheaza transformarile ce vor fi aplicate imaginilor
        
    def __len__(self): #metoda ce va returna nr total de exemplare din setul de date, care este egal cu lungimea lui 'self.states'
        return len(self.states)
    
    def __getitem__(self, idx): #metoda asta va incarca si returna un exemplar din setul de date in fct de index 'idx'
        patient_id = self.states.patient_id.values[idx] #in functie de idx va returna info cum ar fi 'patient_id'
        x_coord = self.states.x.values[idx] # coordonatele x, y
        y_coord = self.states.y.values[idx] 
        image_path = self.states.path.values[idx] #path ul spre imagine
        image = Image.open(image_path) #deschide imaginea specificata prin 'image_path'
        image = image.convert('RGB') #o converteste la formatul RGB si o stocheaza in variabila 'image'
        
        if self.transform: #daca transformarile sunt specificate
            image = self.transform(image) #aplica transformarile la imagine
            
        if "target" in self.states.columns.values: #verifica daca DataFrame contine o coloana numita 'target'
            target = int(self.states.target.values[idx]) #daca da, atunci recupereaza val pt acel sample
        else:
            target = None #daca nu, atunci seteaza 'target' a 'None'
            #se va returna un dictionar ce contine vectorul imaginii, targetul, id ul si coordonatele x, y
        return {"image": image,
                "label": target,
                "patient_id": patient_id,
                "x": x_coord,
                "y": y_coord}


def my_transform(key="train", plot=False):
    train_sequence = [transforms.Resize((50,50)), #pt train se va face o redimensionare la 50x50 si rotiri random orizontale si verticale
                      transforms.ToTensor(), #se converteste imaginea la tensori (adica un fel de vectori) PyTorch
                      transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])] #se normalizeaza valorile tensorilor folosind o medie pecifica si o deviatie standard

    dev_sequence = [transforms.Resize((50,50)), # redimensionare la 50x50 si rotiri random orizontale si verticale
                      transforms.ToTensor(), #se converteste imaginea la tensori (adica un fel de vectori) PyTorch
                      transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])] #se normalizeaza valorile tensorilor folosind o medie pecifica si o deviatie standard
   
    test_sequence = [transforms.Resize((50,50)), #redimensionare
                      transforms.ToTensor(), #se converteste imaginea la tensori (adica un fel de vectori) PyTorch
                      transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])] #se normalizeaza valorile tensorilor folosind o medie pecifica si o deviatie standard
        
    data_transforms = {'train': transforms.Compose(train_sequence), 'dev': transforms.Compose(dev_sequence), 'test': transforms.Compose(test_sequence)}
    return data_transforms[key] #se va returna un dictionar 'data_transforms' ce contine secventele de transformare compuse bazate pe 'key'

#valorile [0.485, 0.456, 0.406], [0.229, 0.224, 0.225] sunt parametrii standard de normalizare folositi pentru modele pre-antrenate


#se face un obiect numit 'train_dataset' ce foloseste DF ul de training 'train_df', specifica trasformarile ce vor fi folosite (functia my_transform) cu cuvantul cheie (key='train')
#se va folosi pt antrenarea modelelor de machine learning
train_dataset = BreastCancerDataset(train_df, transform=my_transform(key="train")) 

#se face un obiect ce foloseste DF ul 'dev_df', specifica trasformarile ce vor fi aplicate cu key='val'
#se va folosi pt a evalua performantele modelului in timpul dezvoltarii sau validarii
dev_dataset = BreastCancerDataset(dev_df, transform=my_transform(key="dev"))

#se face un obiect ce foloseste DF ul 'test_df', cu transformarile specifice pt key='val'
#se va folosi pt evaluarea performantei finale a modelului antrenat pe date nevazute
test_dataset = BreastCancerDataset(test_df, transform=my_transform(key="test"))


#se face un dictionar unde key urile sunt stringuri ce reprezinta tipul dataset urilor (ex 'dev') si valorile corespunzatoare lor (ex 'dev_dataset')
image_datasets = {"train": train_dataset, "dev": dev_dataset, "test": test_dataset}

#se face un dictionar unde key urile sunt tipul seturilor de date (ex'dev') si valorile sunt lungimile (nr de exemplare) din fiecare dataset
#este folosita o intelegere a dictionarului pt a itera prin fiecare set de date  si pt a calcula lungimea fiecaruia
dataset_sizes = {x: len(image_datasets[x]) for x in ["train", "dev", "test"]}


#CREATING PyTorch DATALOADERS

#dataloader pt setul de training
train_dataloader = DataLoader(train_dataset,batch_size=BATCH_SIZE,shuffle=True,drop_last=True)

#dataloader pt setul de validare(dev)
dev_dataloader = DataLoader(dev_dataset,batch_size=BATCH_SIZE,shuffle=False, drop_last=True)

#dataloader pt setul de test
test_dataloader = DataLoader(test_dataset,batch_size=BATCH_SIZE,shuffle=False,drop_last=False)

#batch_size = nr de exemplare/lot de incarcat
#shuffle = True : amesteca datele la fiecare epoca inainte de a crea loturi sa introduca aleatoritate si pt a preveni adaptarea modelului la ordinea datelor
#shuffle = False : dezactiveaza amestecarea datelor din moment ce nu se mai folosesc date de validare pt antrenare, si urmarim un comportament deterministic pe perioada validarii
#drop_last = True : arunca ultimul lot incomplet daca dimensiunea setului de date nu e divizibila cu dimensiunea lotului
#drop_last = False : pastreaza ultimul loc incomplet


#organizam dataloaders de mai sus intr un dictonar pentru a i putea accesa mai usor folosind cuvintele cheie specifice fiecaruia
dataloaders = {"train": train_dataloader, "dev": dev_dataloader, "test": test_dataloader}


#verificam cate loturi sunt in fiecare dataloader
#asa ne putem face o idee despre cate loturi vor fi procesate in timpul fiecarei faze
print(len(dataloaders["train"]), len(dataloaders["dev"]), len(dataloaders["test"]))

5911 1273 1489


#Defining the model structure 

device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
device

device(type='cpu')


#MODELE: 512, 1024, SOFTMAX


#model relu512 
model_relu512 = torchvision.models.resnet18(pretrained=False)
if run_training_relu512:
    model_relu512.load_state_dict(torch.load("../input/pretrained-pytorch-models/resnet18-5c106cde.pth"))
num_features = model_relu512.fc.in_features
    
model_relu512.fc = nn.Sequential( #se modifica layer ul FC(fully connect), va fi inlocuit cu o secventa noua:
    nn.Linear(num_features, 512), # un layer fully connected cu 'num_features' input features si 512 otput features
    nn.ReLU(), #ReLU activation
    nn.BatchNorm1d(512), #normalizarea lotului
    nn.Dropout(0.15), #un dropout cu rata de 0.15
    #inca un fully connected layer
    nn.Linear(512, 256), #512 input features si 256 output features
    nn.ReLU(),
    nn.BatchNorm1d(256),
    nn.Dropout(0.15),
    #ultimul fully connected layes cu 256 input ftrs si 'num_classes'(adica 2) output ftrs
    nn.Linear(256, NUM_CLASSES))

def init_weights(m): #o functie definita ca sa initializeze weight urile layer ului liniar 
    if type(m) == nn.Linear:
        torch.nn.init.xavier_uniform_(m.weight) #este folosita initializarea Xavier
        m.bias.data.fill_(0.01)

model_relu512.apply(init_weights) #aplicarea functiei
model_relu512 = model_relu512.to(device) #modelul este mutat pe device ul specificat


#model relu1024
model_relu1024 = torchvision.models.resnet18(pretrained=False)
if run_training_relu1024:
    model_relu1024.load_state_dict(torch.load("../input/pretrained-pytorch-models/resnet18-5c106cde.pth"))
num_features = model_relu1024.fc.in_features
    
model_relu1024.fc = nn.Sequential( #se modifica layer ul FC(fully connect), va fi inlocuit cu o secventa noua:
    nn.Linear(num_features, 1024), # un layer fully connected cu 'num_features' input features si 512 otput features
    nn.ReLU(), #ReLU activation
    nn.BatchNorm1d(1024), #normalizarea lotului
    nn.Dropout(0.15), #un dropout cu rata de 0.15
    #inca un fully connected layer
    nn.Linear(1024, 256), #512 input features si 256 output features
    nn.ReLU(),
    nn.BatchNorm1d(256),
    nn.Dropout(0.15),
    #ultimul fully connected layes cu 256 input ftrs si 'num_classes'(adica 2) output ftrs
    nn.Linear(256, NUM_CLASSES))

def init_weights(m): #o functie definita ca sa initializeze weight urile layer ului liniar 
    if type(m) == nn.Linear:
        torch.nn.init.xavier_uniform_(m.weight) #este folosita initializarea Xavier
        m.bias.data.fill_(0.01)

model_relu1024.apply(init_weights) #aplicarea functiei
model_relu1024 = model_relu1024.to(device) #modelul este mutat pe device ul specificat


#model softmax

model_softmax = torchvision.models.resnet18(pretrained=False)
if run_training_softmax:
    model_softmax.load_state_dict(torch.load("../input/pretrained-pytorch-models/resnet18-5c106cde.pth"))
num_features = model_softmax.fc.in_features

model_softmax.fc = nn.Sequential(
    nn.Linear(num_features, 1024),
    nn.ReLU(),
    nn.BatchNorm1d(1024),
    nn.Dropout(0.15),
    nn.Linear(1024, 256),
    nn.ReLU(),  # Păstrează ReLU aici
    nn.BatchNorm1d(256),
    nn.Dropout(0.15),
    nn.Linear(256, NUM_CLASSES),
    nn.Softmax(dim=1))  # Adaugă Softmax doar pe ultimul strat

def init_weights(m): #o functie definita ca sa initializeze weight urile layer ului liniar 
    if type(m) == nn.Linear:
        torch.nn.init.xavier_uniform_(m.weight) #este folosita initializarea Xavier
        m.bias.data.fill_(0.01)

model_softmax.apply(init_weights) #aplicarea functiei
model_softmax = model_softmax.to(device) #modelul este mutat pe device ul specificat


#Setting up the loss function

#se va face un class weight bazat pe distribuirea claselor in datele de antrenare
#are nevoe de 2 parametrii:
#y=target din training data
#class_weight=specifica strategia pt calcularea class weights urilor
#in cazul nostru 'balanced' indica faptul ca class weights urile ar trebui sa fie invers proportionale cu frecventa claselor in datele de intrare
# ..unique() = furnizează etichetele de clasă unice prezente în datele de antrenament
weights = compute_class_weight(y=train_df.target.values, class_weight="balanced", classes=train_df.target.unique())    
class_weights = torch.FloatTensor(weights) 
if device.type=="cuda":
    class_weights = class_weights.cuda()
print(class_weights)

tensor([0.6973, 1.7673])


criterion = nn.CrossEntropyLoss(weight=class_weights) 
#prin setarea parametrului 'weights' se asigneaza diferite weight uri claselor in functie de frecventele lor in setul de date


#BUILDING THE TRAINING LOOPS


#RELU 512


def train_loop_relu512(model_relu512, criterion, optimizer, lr_find=False, scheduler=None, num_epochs = 25, lam=0.0):
    since = time.time()
    if lr_find:
        phases = ["train"]
    else:
        phases = ["train", "dev", "test"]
    
    best_model_wts = copy.deepcopy(model_relu512.state_dict())
    best_acc = 0.0
    
    loss_dict_relu512 = {"train": [], "dev": [], "test": []}
    acc_dict_relu512 = {"train": [], "dev": [], "test": []}
    lam_tensor = torch.tensor(lam, device=device)
    
    running_loss_dict_relu512 = {"train": [], "dev": [], "test": []}
    running_acc_dict_relu512 = {"train": [], "dev": [], "test": []}
    
    lr_find_loss = []
    lr_find_lr = []
    smoothing = 0.2
    
    for epoch in range(num_epochs): #se va executa de atatea ori, cate epoci am declarat, pentru fiecare faza(train, dev, test)
        print('Epoch {}/{}'.format(epoch, num_epochs - 1))
        print('-' * 10)
        
        for phase in phases: #setam modelul pe modul de train sau eval
            if phase == "train":
                model_relu512.train()
            else:
                model_relu512.eval()

            running_loss = 0.0
            running_corrects = 0
            
            tk0 = tqdm(dataloaders[phase], total=int(len(dataloaders[phase])))

            counter = 0
            for bi, d in enumerate(tk0):
                inputs = d["image"]
                labels = d["label"]
                inputs = inputs.to(device, dtype=torch.float)
                labels = labels.to(device, dtype=torch.long)
                
                # zero the parameter gradients
                optimizer.zero_grad()
                
                # forward
                # track history if only in train
                
                with torch.set_grad_enabled(phase == 'train'):
                    outputs = model_relu512(inputs)
                    _, preds = torch.max(outputs, 1)
                    loss = criterion(outputs, labels)
                
                    # backward + optimize only if in training phaphase
                    if phase == 'train':
                        loss.backward()            
                        optimizer.step()
                        # cyclical lr schedule is invoked after each batch
                        if scheduler is not None:
                            scheduler.step() 
                            if lr_find:
                                lr_step = optimizer.state_dict()["param_groups"][0]["lr"]
                                lr_find_lr.append(lr_step)
                                if counter==0:
                                    lr_find_loss.append(loss.item())
                                else:
                                    smoothed_loss = smoothing  * loss.item() + (1 - smoothing) * lr_find_loss[-1]
                                    lr_find_loss.append(smoothed_loss)
                            
                # statistics
                running_loss += loss.item() * inputs.size(0)
                running_corrects += torch.sum(preds == labels.data)                      
     
                counter += 1
                
                
                tk0.set_postfix({'loss': running_loss / (counter * dataloaders[phase].batch_size),
                                 'accuracy': running_corrects.double() / (counter * dataloaders[phase].batch_size)})
                running_loss_dict_relu512[phase].append(running_loss / (counter * dataloaders[phase].batch_size))
                running_acc_dict_relu512[phase].append(running_corrects / (counter * dataloaders[phase].batch_size))
                                              
            epoch_loss = running_loss / dataset_sizes[phase]
            loss_dict_relu512[phase].append(epoch_loss)
            epoch_acc = running_corrects.double() / dataset_sizes[phase]
            acc_dict_relu512[phase].append(epoch_acc)
            
            print('{} Loss: {:.4f} Acc: {:.4f}'.format(
                phase, epoch_loss, epoch_acc))
            
            # deep copy the model
            if phase == 'dev' and epoch_acc > best_acc:
                best_acc = epoch_acc
                best_model_wts = copy.deepcopy(model_relu512.state_dict())
        print()
        
    time_elapsed = time.time() - since
    print('Training complete in {:.0f}m {:.0f}s'.format(
    time_elapsed // 60, time_elapsed % 60))
    print('Best val Acc: {:4f}'.format(best_acc))              
    
    # load best model weights
    model_relu512.load_state_dict(best_model_wts)
    results_relu512 = {"model_relu512": model_relu512,
               "loss_dict_relu512": loss_dict_relu512,
               "acc_dict_relu512": acc_dict_relu512,
               "running_loss_dict_relu512": running_loss_dict_relu512,
               "running_acc_dict_relu512": running_acc_dict_relu512,
               "lr_find": {"lr": lr_find_lr, "loss": lr_find_loss}}
    return results_relu512


#RELU 1024


def train_loop_relu1024(model_relu1024, criterion, optimizer, lr_find=False, scheduler=None, num_epochs = 25, lam=0.0):
    since = time.time()
    if lr_find:
        phases = ["train"]
    else:
        phases = ["train", "dev", "test"]
    
    best_model_wts = copy.deepcopy(model_relu1024.state_dict())
    best_acc = 0.0
    
    loss_dict_relu1024 = {"train": [], "dev": [], "test": []}
    acc_dict_relu1024 = {"train": [], "dev": [], "test": []}
    lam_tensor = torch.tensor(lam, device=device)
    
    running_loss_dict_relu1024 = {"train": [], "dev": [], "test": []}
    running_acc_dict_relu1024 = {"train": [], "dev": [], "test": []}
    
    lr_find_loss = []
    lr_find_lr = []
    smoothing = 0.2
    
    for epoch in range(num_epochs): #se va executa de atatea ori, cate epoci am declarat, pentru fiecare faza(train, dev, test)
        print('Epoch {}/{}'.format(epoch, num_epochs - 1))
        print('-' * 10)
        
        for phase in phases: #setam modelul pe modul de train sau eval
            if phase == "train":
                model_relu1024.train()
            else:
                model_relu1024.eval()

            running_loss = 0.0
            running_corrects = 0
            
            tk0 = tqdm(dataloaders[phase], total=int(len(dataloaders[phase])))

            counter = 0
            for bi, d in enumerate(tk0):
                inputs = d["image"]
                labels = d["label"]
                inputs = inputs.to(device, dtype=torch.float)
                labels = labels.to(device, dtype=torch.long)
                
                # zero the parameter gradients
                optimizer.zero_grad()
                
                # forward
                # track history if only in train
                
                with torch.set_grad_enabled(phase == 'train'):
                    outputs = model_relu1024(inputs)
                    _, preds = torch.max(outputs, 1)
                    loss = criterion(outputs, labels)
                
                    # backward + optimize only if in training phaphase
                    if phase == 'train':
                        loss.backward()
                        optimizer.step()
                        # cyclical lr schedule is invoked after each batch
                        if scheduler is not None:
                            scheduler.step() 
                            if lr_find:
                                lr_step = optimizer.state_dict()["param_groups"][0]["lr"]
                                lr_find_lr.append(lr_step)
                                if counter==0:
                                    lr_find_loss.append(loss.item())
                                else:
                                    smoothed_loss = smoothing  * loss.item() + (1 - smoothing) * lr_find_loss[-1]
                                    lr_find_loss.append(smoothed_loss)
                            
                # statistics
                running_loss += loss.item() * inputs.size(0)
                running_corrects += torch.sum(preds == labels.data)                      
     
                counter += 1
                
                
                tk0.set_postfix({'loss': running_loss / (counter * dataloaders[phase].batch_size),
                                 'accuracy': running_corrects.double() / (counter * dataloaders[phase].batch_size)})
                running_loss_dict_relu1024[phase].append(running_loss / (counter * dataloaders[phase].batch_size))
                running_acc_dict_relu1024[phase].append(running_corrects / (counter * dataloaders[phase].batch_size))
                                              
            epoch_loss = running_loss / dataset_sizes[phase]
            loss_dict_relu1024[phase].append(epoch_loss)
            epoch_acc = running_corrects.double() / dataset_sizes[phase]
            acc_dict_relu1024[phase].append(epoch_acc)
            
            print('{} Loss: {:.4f} Acc: {:.4f}'.format(
                phase, epoch_loss, epoch_acc))
            
            # deep copy the model
            if phase == 'dev' and epoch_acc > best_acc:
                best_acc = epoch_acc
                best_model_wts = copy.deepcopy(model_relu1024.state_dict())
        print()
        
    time_elapsed = time.time() - since
    print('Training complete in {:.0f}m {:.0f}s'.format(
    time_elapsed // 60, time_elapsed % 60))
    print('Best val Acc: {:4f}'.format(best_acc))              
    
    # load best model weights
    model_relu1024.load_state_dict(best_model_wts)
    results_relu1024 = {"model_relu1024": model_relu1024,
               "loss_dict_relu1024": loss_dict_relu1024,
               "acc_dict_relu1024": acc_dict_relu1024,
               "running_loss_dict_relu1024": running_loss_dict_relu1024,
               "running_acc_dict_relu1024": running_acc_dict_relu1024,
               "lr_find": {"lr": lr_find_lr, "loss": lr_find_loss}}
    return results_relu1024


def train_loop_softmax(model_softmax, criterion, optimizer, lr_find=False, scheduler=None, num_epochs = 25, lam=0.0):
    since = time.time()
    if lr_find:
        phases = ["train"]
    else:
        phases = ["train", "dev", "test"]
    
    best_model_wts = copy.deepcopy(model_softmax.state_dict())
    best_acc = 0.0
    
    loss_dict_softmax = {"train": [], "dev": [], "test": []}
    acc_dict_softmax = {"train": [], "dev": [], "test": []}
    lam_tensor = torch.tensor(lam, device=device)
    
    running_loss_dict_softmax = {"train": [], "dev": [], "test": []}
    running_acc_dict_softmax = {"train": [], "dev": [], "test": []}
    
    lr_find_loss = []
    lr_find_lr = []
    smoothing = 0.2
    
    for epoch in range(num_epochs): #se va executa de atatea ori, cate epoci am declarat, pentru fiecare faza(train, dev, test)
        print('Epoch {}/{}'.format(epoch, num_epochs - 1))
        print('-' * 10)
        
        for phase in phases: #setam modelul pe modul de train sau eval
            if phase == "train":
                model_softmax.train()
            else:
                model_softmax.eval()

            running_loss = 0.0
            running_corrects = 0
            
            tk0 = tqdm(dataloaders[phase], total=int(len(dataloaders[phase])))

            counter = 0
            for bi, d in enumerate(tk0):
                inputs = d["image"]
                labels = d["label"]
                inputs = inputs.to(device, dtype=torch.float)
                labels = labels.to(device, dtype=torch.long)
                
                # zero the parameter gradients
                optimizer.zero_grad()
                
                # forward
                # track history if only in train
                
                with torch.set_grad_enabled(phase == 'train'):
                    outputs = model_softmax(inputs)
                    _, preds = torch.max(outputs, 1)
                    loss = criterion(outputs, labels)
                
                    # backward + optimize only if in training phaphase
                    if phase == 'train':
                        loss.backward()
                        optimizer.step()
                        # cyclical lr schedule is invoked after each batch
                        if scheduler is not None:
                            scheduler.step() 
                            if lr_find:
                                lr_step = optimizer.state_dict()["param_groups"][0]["lr"]
                                lr_find_lr.append(lr_step)
                                if counter==0:
                                    lr_find_loss.append(loss.item())
                                else:
                                    smoothed_loss = smoothing  * loss.item() + (1 - smoothing) * lr_find_loss[-1]
                                    lr_find_loss.append(smoothed_loss)
                            
                # statistics
                running_loss += loss.item() * inputs.size(0)
                running_corrects += torch.sum(preds == labels.data)                      
     
                counter += 1
                
                
                tk0.set_postfix({'loss': running_loss / (counter * dataloaders[phase].batch_size),
                                 'accuracy': running_corrects.double() / (counter * dataloaders[phase].batch_size)})
                running_loss_dict_softmax[phase].append(running_loss / (counter * dataloaders[phase].batch_size))
                running_acc_dict_softmax[phase].append(running_corrects / (counter * dataloaders[phase].batch_size))
                                              
            epoch_loss = running_loss / dataset_sizes[phase]
            loss_dict_softmax[phase].append(epoch_loss)
            epoch_acc = running_corrects.double() / dataset_sizes[phase]
            acc_dict_softmax[phase].append(epoch_acc)
            
            print('{} Loss: {:.4f} Acc: {:.4f}'.format(
                phase, epoch_loss, epoch_acc))
            
            # deep copy the model
            if phase == 'dev' and epoch_acc > best_acc:
                best_acc = epoch_acc
                best_model_wts = copy.deepcopy(model_softmax.state_dict())
        print()
        
    time_elapsed = time.time() - since
    print('Training complete in {:.0f}m {:.0f}s'.format(
    time_elapsed // 60, time_elapsed % 60))
    print('Best val Acc: {:4f}'.format(best_acc))              
    
    # load best model weights
    model_softmax.load_state_dict(best_model_wts)
    results_softmax = {"model_softmax": model_softmax,
               "loss_dict_softmax": loss_dict_softmax,
               "acc_dict_softmax": acc_dict_softmax,
               "running_loss_dict_softmax": running_loss_dict_softmax,
               "running_acc_dict_softmax": running_acc_dict_softmax,
               "lr_find": {"lr": lr_find_lr, "loss": lr_find_loss}}
    return results_softmax


#Searching for an optimal cyclical learning rate

start_lr = 1e-6
end_lr = 0.1


#functie ce creaza un learning rate scheduler potrivit pt LR range test sau LR search
def get_lr_search_scheduler(optimizer, min_lr, max_lr, max_iterations):
    # max_iterations should be the number of steps within num_epochs_*epoch_iterations
    # this way the learning rate increases linearily within the period num_epochs*epoch_iterations 
    scheduler = torch.optim.lr_scheduler.CyclicLR(optimizer=optimizer, #foloseste 'CyclicLR' care ajusteaza LR ul intre min si max
                                               base_lr=min_lr,
                                               max_lr=max_lr,
                                               step_size_up=max_iterations, #LR creste si scade apoi linear
                                               step_size_down=max_iterations,
                                               mode="triangular")
    
    return scheduler

#functie ce creaza un learning rate scheduler pentru antrenarea normala
def get_scheduler(optimiser, min_lr, max_lr, stepsize): #stepsize rep nr de iteratii din fiecare ciclu
    # suggested_stepsize = 2*num_iterations_within_epoch
    stepsize_up = int(stepsize/2)
    scheduler = torch.optim.lr_scheduler.CyclicLR(optimizer=optimiser,
                                               base_lr=min_lr,
                                               max_lr=max_lr,
                                               step_size_up=stepsize_up,
                                               step_size_down=stepsize_up,
                                               mode="triangular")
    return scheduler


#FIND LEARNING RATE FOR THE MODELS


if find_learning_rate:
    lr_find_epochs=1
    optimizer = optim.SGD(model_relu512.fc.parameters(), start_lr)
    scheduler = get_lr_search_scheduler(optimizer, start_lr, end_lr, lr_find_epochs*len(train_dataloader))
    #se executa training loop specificand modelul, criteriul, etc. fct asta antreneaza modelul pt o epoca in timp ce monitorizeaza pierderea pt diferite lr uri
    results_relu512 = train_loop_relu512(model_relu512, criterion, optimizer, lr_find=True, scheduler=scheduler, num_epochs=lr_find_epochs)
    lr_find_lr, lr_find_loss = results_relu512["lr_find"]["lr"], results_relu512["lr_find"]["loss"]
    #se creaza un DF numit 'find_lr_df' ce se va salva intr un fisier csv numit 'learning_rate_search_relu512'
    find_lr_df_relu512 = pd.DataFrame(lr_find_loss, columns=["smoothed loss"])
    find_lr_df_relu512.loc[:, "lr"] = lr_find_lr
    find_lr_df_relu512.to_csv("learning_rate_search_relu512.csv", index=False)
else: #daca 'find_learning_rate' e Fals, se va citi fisierul salvat anterior csv; asta permite sa refolosim rezultatele fara sa recompilam cautarea
    find_lr_df_relu512 = pd.read_csv(MODEL_PATH + "learning_rate_search_relu512.csv")

Epoch 0/0
----------

  0%|          | 0/5911 [00:00<?, ?it/s]

train Loss: 0.5752 Acc: 0.7524

Training complete in 37m 58s
Best val Acc: 0.000000


if find_learning_rate:
    lr_find_epochs=1
    optimizer = optim.SGD(model_relu1024.fc.parameters(), start_lr)
    scheduler = get_lr_search_scheduler(optimizer, start_lr, end_lr, lr_find_epochs*len(train_dataloader))
    #se executa training loop specificand modelul, criteriul, etc. fct asta antreneaza modelul pt o epoca in timp ce monitorizeaza pierderea pt diferite lr uri
    results_relu1024 = train_loop_relu1024(model_relu1024, criterion, optimizer, lr_find=True, scheduler=scheduler, num_epochs=lr_find_epochs)
    lr_find_lr, lr_find_loss = results_relu1024["lr_find"]["lr"], results_relu1024["lr_find"]["loss"]
    #se creaza un DF numit 'find_lr_df' ce se va salva intr un fisier csv numit 'learning_rate_search_relu1024'
    find_lr_df_relu1024 = pd.DataFrame(lr_find_loss, columns=["smoothed loss"])
    find_lr_df_relu1024.loc[:, "lr"] = lr_find_lr
    find_lr_df_relu1024.to_csv("learning_rate_search_relu1024.csv", index=False)
else: #daca 'find_learning_rate' e Fals, se va citi fisierul salvat anterior csv; asta permite sa refolosim rezultatele fara sa recompilam cautarea
    find_lr_df_relu1024 = pd.read_csv(MODEL_PATH + "learning_rate_search_relu1024.csv")

Epoch 0/0
----------

  0%|          | 0/5911 [00:00<?, ?it/s]

train Loss: 0.5632 Acc: 0.7604

Training complete in 31m 19s
Best val Acc: 0.000000


if find_learning_rate:
    lr_find_epochs=1
    optimizer = optim.SGD(model_softmax.fc.parameters(), start_lr)
    scheduler = get_lr_search_scheduler(optimizer, start_lr, end_lr, lr_find_epochs*len(train_dataloader))
    #se executa training loop specificand modelul, criteriul, etc. fct asta antreneaza modelul pt o epoca in timp ce monitorizeaza pierderea pt diferite lr uri
    results_softmax = train_loop_softmax(model_softmax, criterion, optimizer, lr_find=True, scheduler=scheduler, num_epochs=lr_find_epochs)
    lr_find_lr, lr_find_loss = results_softmax["lr_find"]["lr"], results_softmax["lr_find"]["loss"]
    #se creaza un DF numit 'find_lr_df' ce se va salva intr un fisier csv numit 'learning_rate_search_softmax'
    find_lr_df_softmax = pd.DataFrame(lr_find_loss, columns=["smoothed loss"])
    find_lr_df_softmax.loc[:, "lr"] = lr_find_lr
    find_lr_df_softmax.to_csv("learning_rate_search_relu512.csv", index=False)
else: #daca 'find_learning_rate' e Fals, se va citi fisierul salvat anterior csv; asta permite sa refolosim rezultatele fara sa recompilam cautarea
    find_lr_df_softmax = pd.read_csv(MODEL_PATH + "learning_rate_search__softmax.csv")

Epoch 0/0
----------

  0%|          | 0/5911 [00:00<?, ?it/s]

train Loss: 0.5422 Acc: 0.7694

Training complete in 36m 3s
Best val Acc: 0.000000


#LEARNING RATE PLOTS


fig, ax = plt.subplots(1,2,figsize=(20,5))

ax[0].plot(find_lr_df_relu512.lr.values)
ax[0].set_xlabel("Steps")
ax[0].set_ylabel("Learning rate")
ax[0].set_title("How the learning rate increases during search")

ax[1].plot(find_lr_df_relu512["smoothed loss"].values)
ax[1].set_xlabel("Steps")
ax[1].set_ylabel("Loss");
ax[1].set_title("How the training loss evolves during search")

plt.figure(figsize=(20,5))
plt.plot(find_lr_df_relu512.lr.values, find_lr_df_relu512["smoothed loss"].values, '-', color="#d83057");
plt.xlabel("Learning rate")
plt.xscale("log")
plt.ylabel("Smoothed Loss")
plt.title("Searching for the optimal learning rate for ReLu512");


fig, ax = plt.subplots(1,2,figsize=(20,5))

ax[0].plot(find_lr_df_relu1024.lr.values)
ax[0].set_xlabel("Steps")
ax[0].set_ylabel("Learning rate")
ax[0].set_title("How the learning rate increases during search")

ax[1].plot(find_lr_df_relu1024["smoothed loss"].values)
ax[1].set_xlabel("Steps")
ax[1].set_ylabel("Loss");
ax[1].set_title("How the training loss evolves during search")

plt.figure(figsize=(20,5))
plt.plot(find_lr_df_relu1024.lr.values, find_lr_df_relu1024["smoothed loss"].values, '-', color="#d83057");
plt.xlabel("Learning rate")
plt.xscale("log")
plt.ylabel("Smoothed Loss")
plt.title("Searching for the optimal learning rate for ReLu1024");


fig, ax = plt.subplots(1,2,figsize=(20,5))

ax[0].plot(find_lr_df_softmax.lr.values)
ax[0].set_xlabel("Steps")
ax[0].set_ylabel("Learning rate")
ax[0].set_title("How the learning rate increases during search")

ax[1].plot(find_lr_df_softmax["smoothed loss"].values)
ax[1].set_xlabel("Steps")
ax[1].set_ylabel("Loss");
ax[1].set_title("How the training loss evolves during search")

plt.figure(figsize=(20,5))
plt.plot(find_lr_df_softmax.lr.values, find_lr_df_softmax["smoothed loss"].values, '-', color="#d83057");
plt.xlabel("Learning rate")
plt.xscale("log")
plt.ylabel("Smoothed Loss")
plt.title("Searching for the optimal learning rate for SoftMax");


start_lr = 1e-6
end_lr = 0.006


#ANTRENARE PE MODELE 512, 1024, SOFTMAX


run_training_relu512 = True

if run_training_relu512: # Restul codului pentru antrenare
    NUM_EPOCHS = 25
    optimizer = optim.SGD(model_relu512.fc.parameters(), lr=0.01)
    scheduler = get_scheduler(optimizer, start_lr, end_lr, 2*NUM_EPOCHS)
    results_relu512 = train_loop_relu512(model_relu512, criterion, optimizer, scheduler=scheduler, num_epochs = NUM_EPOCHS)
    model_relu512, loss_dict_relu512, acc_dict_relu512, running_loss_dict_relu512, running_acc_dict_relu512 = results_relu512["model_relu512"], results_relu512["loss_dict_relu512"], results_relu512["acc_dict_relu512"], results_relu512["running_loss_dict_relu512"], results_relu512["running_acc_dict_relu512"]
    
    if device == "cpu":
        OUTPUT_PATH += ".pth"
    else:
        OUTPUT_PATH += "_cuda.pth"
        
    torch.save(model_relu512.state_dict(), OUTPUT_PATH)
    losses_df_relu512 = pd.DataFrame(loss_dict_relu512["train"],columns=["train"])
    losses_df_relu512.loc[:, "dev"] = loss_dict_relu512["dev"]
    losses_df_relu512.loc[:, "test"] = loss_dict_relu512["test"]
    losses_df_relu512.to_csv("losses_breastcancer_relu512.csv", index=False)
    
    acc_df_relu512 = pd.DataFrame(acc_dict_relu512["train"], columns=["train"])
    acc_df_relu512.loc[:, "dev"] = acc_dict_relu512["dev"]
    acc_df_relu512.loc[:, "test"] = acc_dict_relu512["test"]
    acc_df_relu512.to_csv("accuracies_breastcancer_relu512.csv", index=False)

    running_losses_df_relu512 = pd.DataFrame(running_loss_dict_relu512["train"], columns=["train"])
    running_losses_df_relu512.loc[0:len(running_loss_dict_relu512["dev"])-1, "dev"] = running_loss_dict_relu512["dev"]
    running_losses_df_relu512.loc[0:len(running_loss_dict_relu512["test"])-1, "test"] = running_loss_dict_relu512["test"]
    running_losses_df_relu512.to_csv("running_losses_breastcancer_relu512.csv", index=False)
    
    running_acc_df_relu512 = pd.DataFrame(running_acc_dict_relu512["train"], columns=["train"])
    running_acc_df_relu512.loc[0:len(running_acc_dict_relu512["dev"])-1, "dev"] = running_acc_dict_relu512["dev"]
    running_acc_df_relu512.loc[0:len(running_acc_dict_relu512["test"])-1, "test"] = running_acc_dict_relu512["test"]
    running_acc_df_relu512.to_csv("running_accuracies_breastcancer_relu512.csv", index=False)
    
    
else: # Restul codului pentru încărcarea modelului antrenat
    if device == "cpu":
        load_path = MODEL_PATH + ".pth"
    else:
        load_path = MODEL_PATH + "_cuda.pth"
    model_relu512.load_state_dict(torch.load(load_path, map_location='cpu'))
    model_relu512.eval()
    losses_df_relu512 = pd.read_csv(LOSSES_PATH + "losses_breastcancer_relu512.csv")
    running_losses_df_relu512 = pd.read_csv(LOSSES_PATH + "running_losses_breastcancer_relu512.csv")
    acc_df_relu512 = pd.read_csv(ACC_PATH + "accuracies_breastcancer_relu512.csv")
    running_acc_df_relu512 = pd.read_csv(ACC_PATH + "running_accuracies_breastcancer_relu512.csv")

Epoch 0/24
----------

  0%|          | 0/5911 [00:00<?, ?it/s]

train Loss: 0.5244 Acc: 0.7628

  0%|          | 0/1273 [00:00<?, ?it/s]

dev Loss: 0.4730 Acc: 0.7887

  0%|          | 0/1489 [00:00<?, ?it/s]

test Loss: 0.4282 Acc: 0.8127

Epoch 1/24
----------

  0%|          | 0/5911 [00:00<?, ?it/s]

train Loss: 0.5032 Acc: 0.7748

  0%|          | 0/1273 [00:00<?, ?it/s]


run_training_relu1024 = True

if run_training_relu1024: # Restul codului pentru antrenare
    NUM_EPOCHS = 25
    optimizer = optim.SGD(model_relu1024.fc.parameters(), lr=0.01)
    scheduler = get_scheduler(optimizer, start_lr, end_lr, 2*NUM_EPOCHS)
    results_relu1024 = train_loop_relu1024(model_relu1024, criterion, optimizer, scheduler=scheduler, num_epochs = NUM_EPOCHS)
    model_relu1024, loss_dict_relu1024, acc_dict_relu1024, running_loss_dict_relu1024, running_acc_dict_relu1024 = results_relu1024["model_relu1024"], results_relu1024["loss_dict_relu1024"], results_relu1024["acc_dict_relu1024"], results_relu1024["running_loss_dict_relu1024"], results_relu1024["running_acc_dict_relu1024"]
    
    if device == "cpu":
        OUTPUT_PATH += ".pth"
    else:
        OUTPUT_PATH += "_cuda.pth"
        
    torch.save(model_relu1024.state_dict(), OUTPUT_PATH)
    losses_df_relu1024 = pd.DataFrame(loss_dict_relu1024["train"],columns=["train"])
    losses_df_relu1024.loc[:, "dev"] = loss_dict_relu1024["dev"]
    losses_df_relu1024.loc[:, "test"] = loss_dict_relu1024["test"]
    losses_df_relu1024.to_csv("losses_breastcancer_relu1024.csv", index=False)
    
    acc_df_relu1024 = pd.DataFrame(acc_dict_relu1024["train"], columns=["train"])
    acc_df_relu1024.loc[:, "dev"] = acc_dict_relu1024["dev"]
    acc_df_relu1024.loc[:, "test"] = acc_dict_relu1024["test"]
    acc_df_relu1024.to_csv("accuracies_breastcancer_relu1024.csv", index=False)

    running_losses_df_relu1024 = pd.DataFrame(running_loss_dict_relu1024["train"], columns=["train"])
    running_losses_df_relu1024.loc[0:len(running_loss_dict_relu1024["dev"])-1, "dev"] = running_loss_dict_relu1024["dev"]
    running_losses_df_relu1024.loc[0:len(running_loss_dict_relu1024["test"])-1, "test"] = running_loss_dict_relu1024["test"]
    running_losses_df_relu1024.to_csv("running_losses_breastcancer_relu1024.csv", index=False)
    
    running_acc_df_relu1024 = pd.DataFrame(running_acc_dict_relu1024["train"], columns=["train"])
    running_acc_df_relu1024.loc[0:len(running_acc_dict_relu1024["dev"])-1, "dev"] = running_acc_dict_relu1024["dev"]
    running_acc_df_relu1024.loc[0:len(running_acc_dict_relu1024["test"])-1, "test"] = running_acc_dict_relu1024["test"]
    running_acc_df_relu1024.to_csv("running_accuracies_breastcancer_relu1024.csv", index=False)
    
    
else: # Restul codului pentru încărcarea modelului antrenat
    if device == "cpu":
        load_path = MODEL_PATH + ".pth"
    else:
        load_path = MODEL_PATH + "_cuda.pth"
    model_relu1024.load_state_dict(torch.load(load_path, map_location='cpu'))
    model_relu1024.eval()
    losses_df_relu1024 = pd.read_csv(LOSSES_PATH + "losses_breastcancer_relu1024.csv")
    running_losses_df_relu1024 = pd.read_csv(LOSSES_PATH + "running_losses_breastcancer_relu1024.csv")
    acc_df_relu1024 = pd.read_csv(ACC_PATH + "accuracies_breastcancer_relu1024.csv")
    running_acc_df_relu1024 = pd.read_csv(ACC_PATH + "running_accuracies_breastcancer_relu1024.csv")

Epoch 0/24
----------

  0%|          | 0/5911 [00:00<?, ?it/s]

train Loss: 0.4371 Acc: 0.8066

  0%|          | 0/1273 [00:00<?, ?it/s]

dev Loss: 0.4499 Acc: 0.8002

  0%|          | 0/1489 [00:00<?, ?it/s]

test Loss: 0.3854 Acc: 0.8393

Epoch 1/24
----------

  0%|          | 0/5911 [00:00<?, ?it/s]

train Loss: 0.4367 Acc: 0.8080

  0%|          | 0/1273 [00:00<?, ?it/s]


run_training_softmax = True

if run_training_softmax: # Restul codului pentru antrenare
    NUM_EPOCHS = 25 
    optimizer = optim.SGD(model_softmax.fc.parameters(), lr=0.01)
    scheduler = get_scheduler(optimizer, start_lr, end_lr, 2*NUM_EPOCHS)
    results_softmax = train_loop_softmax(model_softmax, criterion, optimizer, scheduler=scheduler, num_epochs = NUM_EPOCHS)
    #aici am adagat modificari dar nu le-am salvat!
    model_softmax, loss_dict_softmax, acc_dict_softmax, running_loss_dict_softmax, running_acc_dict_softmax = results_softmax["model_softmax"], results_softmax["loss_dict_softmax"], results_softmax["acc_dict_softmax"], results_softmax["running_loss_dict_softmax"], results_softmax["running_acc_dict_softmax"]
    
    if device == "cpu":
        OUTPUT_PATH += ".pth"
    else:
        OUTPUT_PATH += "_cuda.pth"
        
    torch.save(model_softmax.state_dict(), OUTPUT_PATH)
    losses_df_softmax = pd.DataFrame(loss_dict_softmax["train"],columns=["train"])
    losses_df_softmax.loc[:, "dev"] = loss_dict_softmax["dev"]
    losses_df_softmax.loc[:, "test"] = loss_dict_softmax["test"]
    losses_df_softmax.to_csv("losses_breastcancer_softmax1024.csv", index=False)
    
    acc_df_softmax = pd.DataFrame(acc_dict_softmax["train"], columns=["train"])
    acc_df_softmax.loc[:, "dev"] = acc_dict_softmax["dev"]
    acc_df_softmax.loc[:, "test"] = acc_dict_softmax["test"]
    acc_df_softmax.to_csv("accuracies_breastcancer_softmax1024.csv", index=False)

    running_losses_df_softmax = pd.DataFrame(running_loss_dict_softmax["train"], columns=["train"])
    running_losses_df_softmax.loc[0:len(running_loss_dict_softmax["dev"])-1, "dev"] = running_loss_dict_softmax["dev"]
    running_losses_df_softmax.loc[0:len(running_loss_dict_softmax["test"])-1, "test"] = running_loss_dict_softmax["test"]
    running_losses_df_softmax.to_csv("running_losses_breastcancer_softmax1024.csv", index=False)
    
    running_acc_df_softmax = pd.DataFrame(running_acc_dict_softmax["train"], columns=["train"])
    running_acc_df_softmax.loc[0:len(running_acc_dict_softmax["dev"])-1, "dev"] = running_acc_dict_softmax["dev"]
    running_acc_df_softmax.loc[0:len(running_acc_dict_softmax["test"])-1, "test"] = running_acc_dict_softmax["test"]
    running_acc_df_softmax.to_csv("running_accuracies_breastcancer_softmax1025.csv", index=False)
    
    
else: # Restul codului pentru încărcarea modelului antrenat
    if device == "cpu":
        load_path = MODEL_PATH + ".pth"
    else:
        load_path = MODEL_PATH + "_cuda.pth"
    model_softmax.load_state_dict(torch.load(load_path, map_location='cpu'))
    model_softmax.eval()
    losses_df_softmax = pd.read_csv(LOSSES_PATH + "losses_breastcancer_softmax1024.csv")
    running_losses_df_softmax = pd.read_csv(LOSSES_PATH + "running_losses_breastcancer_softmax1024.csv")
    acc_df_softmax = pd.read_csv(ACC_PATH + "accuracies_breastcancer_softmax1024.csv")
    running_acc_df_softmax = pd.read_csv(ACC_PATH + "running_accuracies_breastcancer_softmax1024.csv")

Epoch 0/24
----------

  0%|          | 0/5911 [00:00<?, ?it/s]

train Loss: 0.5316 Acc: 0.7755

  0%|          | 0/1273 [00:00<?, ?it/s]

dev Loss: 0.5038 Acc: 0.8015

  0%|          | 0/1489 [00:00<?, ?it/s]

test Loss: 0.4777 Acc: 0.8305

Epoch 1/24
----------

  0%|          | 0/5911 [00:00<?, ?it/s]

train Loss: 0.5213 Acc: 0.7888

  0%|          | 0/1273 [00:00<?, ?it/s]


#LOSS CONVERGENCE: RELU512

plt.figure(figsize=(20,5))

plt.plot(losses_df_relu512["train"], '-o', label="train", color="#d8638e")
plt.plot(losses_df_relu512["dev"], '-o', label="dev", color="#d83057")

plt.xlabel("Epoch")
plt.ylabel("Weighted x-entropy")
plt.title("Loss change over epoch for ReLu512")
plt.legend();


#LOSS CONVERGENCE: RELU1024

plt.figure(figsize=(20,5))

plt.plot(losses_df_relu1024["train"], '-o', label="train", color="#d8638e")
plt.plot(losses_df_relu1024["dev"], '-o', label="dev", color="#d83057")

plt.xlabel("Epoch")
plt.ylabel("Weighted x-entropy")
plt.title("Loss change over epoch for ReLu1024")
plt.legend();


#LOSS CONVERGENCE: SOFTMAX

plt.figure(figsize=(20,5))

plt.plot(losses_df_softmax["train"], '-o', label="train", color="#d8638e")
plt.plot(losses_df_softmax["dev"], '-o', label="dev", color="#d83057")

plt.xlabel("Epoch")
plt.ylabel("Weighted x-entropy")
plt.title("Loss change over epoch for SoftMax")
plt.legend();


#ACCURACY RELU512

plt.figure(figsize=(20,5))

plt.plot(acc_df_relu512["train"], '-o', label="train", color="#d8638e")
plt.plot(acc_df_relu512["dev"], '-o', label="dev", color="#d83057")

plt.xlabel("Epoch")
plt.ylabel("Weighted x-entropy")
plt.title("Accuracy change over epoch for ReLu512")
plt.legend();


#ACCURACY RELU1024

plt.figure(figsize=(20,5))

plt.plot(acc_df_relu1024["train"], '-o', label="train", color="#d8638e")
plt.plot(acc_df_relu1024["dev"], '-o', label="dev", color="#d83057")

plt.xlabel("Epoch")
plt.ylabel("Weighted x-entropy")
plt.title("Accuracy change over epoch for ReLu1024")
plt.legend();


#ACCURACY SOFTMAX

plt.figure(figsize=(20,5))

plt.plot(acc_df_softmax["train"], '-o', label="train", color="#d8638e")
plt.plot(acc_df_softmax["dev"], '-o', label="dev", color="#d83057")

plt.xlabel("Epoch")
plt.ylabel("Weighted x-entropy")
plt.title("Accuracy change over epoch for SoftMax1024")
plt.legend();


#EVOLUTIE LOSS


#ploturi cu evolutia pierderilor pe parcursul etapelor pentru seturile de date de dev, test si train
#etapa adica pentru fiecare set de date(BATCH sau LOT) care este prezentat modelului
fig, ax = plt.subplots(4,1,figsize=(20,40))
fig.subplots_adjust(hspace=0.5)
line_width = 3.5

ax[0].plot(running_losses_df_relu512["train"], label="train", color="#B5519E", linewidth=line_width)
ax[0].set_xlabel("Step")
ax[0].set_ylabel("Weighted x-entropy")
ax[0].set_title("Loss change over steps in 'TRAIN'")
ax[0].legend();

ax[1].plot(running_losses_df_relu512["train"], label="train", color="#B5519E", linewidth=line_width)
ax[1].set_xlabel("Step")
ax[1].set_ylabel("Weighted x-entropy")
ax[1].set_title("ZOOMED Loss change over steps in 'TRAIN'")
ax[1].set_xlim(-100, 500);
ax[1].legend();

ax[2].plot(running_losses_df_relu512["dev"], label="dev", color="#FF9999", linewidth=line_width)
ax[2].set_xlabel("Step")
ax[2].set_ylabel("Weighted x-entropy")
ax[2].set_title("Loss change over steps in 'DEV'")
ax[2].legend();

ax[3].plot(running_losses_df_relu512["dev"], label="dev", color="#FF9999", linewidth=line_width)
ax[3].set_xlabel("Step")
ax[3].set_ylabel("Weighted x-entropy")
ax[3].set_title("ZOOMED Loss change over steps in 'DEV'")
ax[3].set_xlim(-100, 2500);
ax[3].legend();


#ploturi cu evolutia pierderilor pe parcursul etapelor pentru seturile de date de dev, test si train
#etapa adica pentru fiecare set de date(BATCH sau LOT) care este prezentat modelului
fig, ax = plt.subplots(4,1,figsize=(20,40))
fig.subplots_adjust(hspace=0.5)
line_width = 3.5

ax[0].plot(running_losses_df_relu1024["train"], label="train", color="#B5519E", linewidth=line_width)
ax[0].set_xlabel("Step")
ax[0].set_ylabel("Weighted x-entropy")
ax[0].set_title("Loss change over steps in 'TRAIN'")
ax[0].legend();

ax[1].plot(running_losses_df_relu1024["train"], label="train", color="#B5519E", linewidth=line_width)
ax[1].set_xlabel("Step")
ax[1].set_ylabel("Weighted x-entropy")
ax[1].set_title("ZOOMED Loss change over steps in 'TRAIN'")
ax[1].set_xlim(-100, 500);
ax[1].legend();

ax[2].plot(running_losses_df_relu1024["dev"], label="dev", color="#FF9999", linewidth=line_width)
ax[2].set_xlabel("Step")
ax[2].set_ylabel("Weighted x-entropy")
ax[2].set_title("Loss change over steps in 'DEV'")
ax[2].legend();

ax[3].plot(running_losses_df_relu1024["dev"], label="dev", color="#FF9999", linewidth=line_width)
ax[3].set_xlabel("Step")
ax[3].set_ylabel("Weighted x-entropy")
ax[3].set_title("ZOOMED Loss change over steps in 'DEV'")
ax[3].set_xlim(-100, 2500);
ax[3].legend();


#ploturi cu evolutia pierderilor pe parcursul etapelor pentru seturile de date de dev, test si train
#etapa adica pentru fiecare set de date(BATCH sau LOT) care este prezentat modelului
fig, ax = plt.subplots(4,1,figsize=(20,40))
fig.subplots_adjust(hspace=0.5)
line_width = 3.5

ax[0].plot(running_losses_df_softmax["train"], label="train", color="#B5519E", linewidth=line_width)
ax[0].set_xlabel("Step")
ax[0].set_ylabel("Weighted x-entropy")
ax[0].set_title("Loss change over steps in 'TRAIN'")
ax[0].legend();

ax[1].plot(running_losses_df_softmax["train"], label="train", color="#B5519E", linewidth=line_width)
ax[1].set_xlabel("Step")
ax[1].set_ylabel("Weighted x-entropy")
ax[1].set_title("ZOOMED Loss change over steps in 'TRAIN'")
ax[1].set_xlim(-100, 500);
ax[1].legend();

ax[2].plot(running_losses_df_softmax["dev"], label="dev", color="#FF9999", linewidth=line_width)
ax[2].set_xlabel("Step")
ax[2].set_ylabel("Weighted x-entropy")
ax[2].set_title("Loss change over steps in 'DEV'")
ax[2].legend();

ax[3].plot(running_losses_df_softmax["dev"], label="dev", color="#FF9999", linewidth=line_width)
ax[3].set_xlabel("Step")
ax[3].set_ylabel("Weighted x-entropy")
ax[3].set_title("ZOOMED Loss change over steps in 'DEV'")
ax[3].set_xlim(-100, 2500);
ax[3].legend();


#EVOLUTIE ACC


fig, ax = plt.subplots(4, 1, figsize=(20, 40))
fig.subplots_adjust(hspace=0.5)

# Setează grosimea liniilor dorită
line_width = 3.5

ax[0].plot(running_acc_df_relu512["train"], label="train", color="#B5519E", linewidth=line_width)
ax[0].set_xlabel("Step")
ax[0].set_ylabel("Weighted x-entropy")
ax[0].set_title("Accuracy change over steps in 'TRAIN'")
ax[0].legend()

ax[1].plot(running_acc_df_relu512["train"], label="train", color="#B5519E", linewidth=line_width)
ax[1].set_xlabel("Step")
ax[1].set_ylabel("Weighted x-entropy")
ax[1].set_title("ZOOMED Accuracy change over steps in 'TRAIN'")
ax[1].set_xlim(-100, 500)
ax[1].legend()

ax[2].plot(running_acc_df_relu512["dev"], label="dev", color="#FF9999", linewidth=line_width)
ax[2].set_xlabel("Step")
ax[2].set_ylabel("Weighted x-entropy")
ax[2].set_title("Accuracy change over steps in 'DEV'")
ax[2].legend()

ax[3].plot(running_acc_df_relu512["dev"], label="dev", color="#FF9999", linewidth=line_width)
ax[3].set_xlabel("Step")
ax[3].set_ylabel("Weighted x-entropy")
ax[3].set_title("ZOOMED Accuracy change over steps in 'DEV'")
ax[3].set_xlim(-100, 2500)
ax[3].legend()

plt.tight_layout()
plt.show()


fig, ax = plt.subplots(4, 1, figsize=(20, 40))
fig.subplots_adjust(hspace=0.5)

# Setează grosimea liniilor dorită
line_width = 3.5

ax[0].plot(running_acc_df_relu1024["train"], label="train", color="#B5519E", linewidth=line_width)
ax[0].set_xlabel("Step")
ax[0].set_ylabel("Weighted x-entropy")
ax[0].set_title("Accuracy change over steps in 'TRAIN'")
ax[0].legend()

ax[1].plot(running_acc_df_relu1024["train"], label="train", color="#B5519E", linewidth=line_width)
ax[1].set_xlabel("Step")
ax[1].set_ylabel("Weighted x-entropy")
ax[1].set_title("ZOOMED Accuracy change over steps in 'TRAIN'")
ax[1].set_xlim(-100, 500)
ax[1].legend()

ax[2].plot(running_acc_df_relu1024["dev"], label="dev", color="#FF9999", linewidth=line_width)
ax[2].set_xlabel("Step")
ax[2].set_ylabel("Weighted x-entropy")
ax[2].set_title("Accuracy change over steps in 'DEV'")
ax[2].legend()

ax[3].plot(running_acc_df_relu1024["dev"], label="dev", color="#FF9999", linewidth=line_width)
ax[3].set_xlabel("Step")
ax[3].set_ylabel("Weighted x-entropy")
ax[3].set_title("ZOOMED Accuracy change over steps in 'DEV'")
ax[3].set_xlim(-100, 2500)
ax[3].legend()

plt.tight_layout()
plt.show()


fig, ax = plt.subplots(4, 1, figsize=(20, 40))
fig.subplots_adjust(hspace=0.5)

# Setează grosimea liniilor dorită
line_width = 3.5

ax[0].plot(running_acc_df_softmax["train"], label="train", color="#B5519E", linewidth=line_width)
ax[0].set_xlabel("Step")
ax[0].set_ylabel("Weighted x-entropy")
ax[0].set_title("Accuracy change over steps in 'TRAIN'")
ax[0].legend()

ax[1].plot(running_acc_df_softmax["train"], label="train", color="#B5519E", linewidth=line_width)
ax[1].set_xlabel("Step")
ax[1].set_ylabel("Weighted x-entropy")
ax[1].set_title("ZOOMED Accuracy change over steps in 'TRAIN'")
ax[1].set_xlim(-100, 500)
ax[1].legend()

ax[2].plot(running_acc_df_softmax["dev"], label="dev", color="#FF9999", linewidth=line_width)
ax[2].set_xlabel("Step")
ax[2].set_ylabel("Weighted x-entropy")
ax[2].set_title("Accuracy change over steps in 'DEV'")
ax[2].legend()

ax[3].plot(running_acc_df_softmax["dev"], label="dev", color="#FF9999", linewidth=line_width)
ax[3].set_xlabel("Step")
ax[3].set_ylabel("Weighted x-entropy")
ax[3].set_title("ZOOMED Accuracy change over steps in 'DEV'")
ax[3].set_xlim(-100, 2500)
ax[3].legend()

plt.tight_layout()
plt.show()


#EVALUARE MODELE


def evaluate_model_relu512(model_relu512, test_dataloader, device):
    predictions_df_relu512 = pd.DataFrame(columns=["proba", "true", "predicted", "x", "y", "patient_id"])
    all_preds = []
    all_targets = []
    model_relu512.eval()
    
    with torch.no_grad():
        for i, data in enumerate(test_dataloader):
            inputs = data["image"].to(device)
            labels = data["label"].to(device)
            patient_ids = data["patient_id"]
            x_coords = data["x"]
            y_coords = data["y"]
            
            outputs = model_relu512(inputs)
            _, preds = torch.max(outputs, 1)
            probs = expit(outputs.cpu().numpy())[:, 1]  # Applying sigmoid to get probabilities
            
            batch_size = inputs.size(0)
            for j in range(batch_size):
                predictions_df_relu512.loc[len(predictions_df_relu512)] = {
                    "proba": probs[j].item(),
                    "true": labels[j].item(),
                    "predicted": preds[j].item(),
                    "x": x_coords[j].item(),
                    "y": y_coords[j].item(),
                    "patient_id": patient_ids[j]
                }
    
    return predictions_df_relu512


def evaluate_model_relu1024(model_relu1024, test_dataloader, device):
    predictions_df_relu1024 = pd.DataFrame(columns=["proba", "true", "predicted", "x", "y", "patient_id"])
    all_preds = []
    all_targets = []
    model_relu1024.eval()
    
    with torch.no_grad():
        for i, data in enumerate(test_dataloader):
            inputs = data["image"].to(device)
            labels = data["label"].to(device)
            patient_ids = data["patient_id"]
            x_coords = data["x"]
            y_coords = data["y"]
            
            outputs = model_relu1024(inputs)
            _, preds = torch.max(outputs, 1)
            probs = expit(outputs.cpu().numpy())[:, 1]  # Applying sigmoid to get probabilities
            
            batch_size = inputs.size(0)
            for j in range(batch_size):
                predictions_df_relu1024.loc[len(predictions_df_relu1024)] = {
                    "proba": probs[j].item(),
                    "true": labels[j].item(),
                    "predicted": preds[j].item(),
                    "x": x_coords[j].item(),
                    "y": y_coords[j].item(),
                    "patient_id": patient_ids[j]
                }
    
    return predictions_df_relu1024


def evaluate_model_softmax(model_softmax, test_dataloader, device):
    predictions_df_softmax = pd.DataFrame(columns=["proba", "true", "predicted", "x", "y", "patient_id"])
    all_preds = []
    all_targets = []
    model_softmax.eval()
    
    with torch.no_grad():
        for i, data in enumerate(test_dataloader):
            inputs = data["image"].to(device)
            labels = data["label"].to(device)
            patient_ids = data["patient_id"]
            x_coords = data["x"]
            y_coords = data["y"]
            
            outputs = model_softmax(inputs)
            _, preds = torch.max(outputs, 1)
            probs = expit(outputs.cpu().numpy())[:, 1]  # Applying sigmoid to get probabilities
            
            batch_size = inputs.size(0)
            for j in range(batch_size):
                predictions_df_softmax.loc[len(predictions_df_softmax)] = {
                    "proba": probs[j].item(),
                    "true": labels[j].item(),
                    "predicted": preds[j].item(),
                    "x": x_coords[j].item(),
                    "y": y_coords[j].item(),
                    "patient_id": patient_ids[j]
                }
    
    return predictions_df_softmax


#EVALUARE SI SALVARE


# Evaluare model și calcul scoruri
predictions_df_relu512 = evaluate_model_relu512(model_relu512, test_dataloader, device)
# Salvare în fișier CSV
predictions_df_relu512.to_csv("evaluare_model_relu512.csv", index=False)
# Afișare primele 10 rânduri din predictions_df
print(predictions_df_relu512.head(10))

      proba  true  predicted     x     y patient_id
0  0.688273     0          1  1001  1301      10260
1  0.831958     0          1  1001  1351      10260
2  0.504478     0          1  1001  1401      10260
3  0.409591     0          1  1001  1451      10260
4  0.665545     0          1  1001  1501      10260
5  0.724151     0          1  1001  1551      10260
6  0.267211     0          0  1001  1601      10260
7  0.126317     0          0  1001   201      10260
8  0.158580     0          0  1001   251      10260
9  0.363323     0          0  1001   301      10260


# Evaluare model și calcul scoruri
predictions_df_relu1024 = evaluate_model_relu1024(model_relu1024, test_dataloader, device)
# Salvare în fișier CSV
predictions_df_relu1024.to_csv("evaluare_model_relu1024.csv", index=False)
# Afișare primele 10 rânduri din predictions_df
print(predictions_df_relu1024.head(10))

      proba  true  predicted     x     y patient_id
0  0.446599     0          0  1001  1301      10260
1  0.770952     0          1  1001  1351      10260
2  0.444892     0          0  1001  1401      10260
3  0.410939     0          0  1001  1451      10260
4  0.615124     0          1  1001  1501      10260
5  0.502684     0          0  1001  1551      10260
6  0.104741     0          0  1001  1601      10260
7  0.371580     0          0  1001   201      10260
8  0.268685     0          0  1001   251      10260
9  0.062039     0          0  1001   301      10260


# Evaluare model și calcul scoruri
predictions_df_softmax = evaluate_model_softmax(model_softmax, test_dataloader, device)
# Salvare în fișier CSV
predictions_df_softmax.to_csv("evaluare_model_softmax.csv", index=False)
# Afișare primele 10 rânduri din predictions_df
print(predictions_df_softmax.head(10))

      proba  true  predicted     x     y patient_id
0  0.728744     0          1  1001  1301      10260
1  0.725985     0          1  1001  1351      10260
2  0.625407     0          1  1001  1401      10260
3  0.719666     0          1  1001  1451      10260
4  0.730879     0          1  1001  1501      10260
5  0.693481     0          1  1001  1551      10260
6  0.500339     0          0  1001  1601      10260
7  0.500000     0          0  1001   201      10260
8  0.552584     0          0  1001   251      10260
9  0.500615     0          0  1001   301      10260


#CALCUL SCORURI


# Funcție pentru calculul scorurilor
def calculate_metrics(y_true, y_pred):
    precision = precision_score(y_true, y_pred)
    recall = recall_score(y_true, y_pred)
    f1 = f1_score(y_true, y_pred)
    accuracy = accuracy_score(y_true, y_pred)
    return precision, recall, f1, accuracy

precision_relu512, recall_relu512, f1_relu512, accuracy_relu512 = calculate_metrics(predictions_df_relu512["true"], predictions_df_relu512["predicted"])
precision_relu1024, recall_relu1024, f1_relu1024, accuracy_relu1024 = calculate_metrics(predictions_df_relu1024["true"], predictions_df_relu1024["predicted"])
precision_softmax, recall_softmax, f1_softmax, accuracy_softmax = calculate_metrics(predictions_df_softmax["true"], predictions_df_softmax["predicted"])

# Afișare scoruri pentru fiecare model
print('Model ReLU 512:')
print(f'Precision: {precision_relu512:.4f}')
print(f'Recall: {recall_relu512:.4f}')
print(f'F1 Score: {f1_relu512:.4f}')
print(f'Accuracy: {accuracy_relu512:.4f}')
print()

print('Model ReLU 1024:')
print(f'Precision: {precision_relu1024:.4f}')
print(f'Recall: {recall_relu1024:.4f}')
print(f'F1 Score: {f1_relu1024:.4f}')
print(f'Accuracy: {accuracy_relu1024:.4f}')
print()

print('Model Softmax:')
print(f'Precision: {precision_softmax:.4f}')
print(f'Recall: {recall_softmax:.4f}')
print(f'F1 Score: {f1_softmax:.4f}')
print(f'Accuracy: {accuracy_softmax:.4f}')

Model ReLU 512:
Precision: 0.6898
Recall: 0.7517
F1 Score: 0.7195
Accuracy: 0.8354

Model ReLU 1024:
Precision: 0.7194
Recall: 0.7291
F1 Score: 0.7242
Accuracy: 0.8441

Model Softmax:
Precision: 0.7007
Recall: 0.7461
F1 Score: 0.7227
Accuracy: 0.8393


#matrice de confuzie
from matplotlib.colors import LinearSegmentedColormap

pinky_cmap = LinearSegmentedColormap.from_list("pinky", ["#ffb6c1", "#FF80A6", "#FF05A3", "#FF99CC"])


class_names = [0, 1] 
def plot_confusion_matrix_relu512(y_true, y_pred, class_names):
    cm = confusion_matrix(y_true, y_pred)
    plt.figure(figsize=(8, 6))
    sns.heatmap(cm, annot=True, fmt="d", cmap=pinky_cmap, xticklabels=class_names, yticklabels=class_names)
    plt.xlabel("Predicted")
    plt.ylabel("True")
    plt.title("Confusion Matrix for ReLu512")
    plt.show()

# Desenare matrice de confuzie
plot_confusion_matrix_relu512(predictions_df_relu512["true"], predictions_df_relu512["predicted"], class_names)


class_names = [0, 1] 
def plot_confusion_matrix_relu1024(y_true, y_pred, class_names):
    cm = confusion_matrix(y_true, y_pred)
    plt.figure(figsize=(8, 6))
    sns.heatmap(cm, annot=True, fmt="d", cmap=pinky_cmap, xticklabels=class_names, yticklabels=class_names)
    plt.xlabel("Predicted")
    plt.ylabel("True")
    plt.title("Confusion Matrix for Relu1024")
    plt.show()

# Desenare matrice de confuzie
plot_confusion_matrix_relu1024(predictions_df_relu1024["true"], predictions_df_relu1024["predicted"], class_names)


class_names = [0, 1] 
def plot_confusion_matrix_softmax(y_true, y_pred, class_names):
    cm = confusion_matrix(y_true, y_pred)
    plt.figure(figsize=(8, 6))
    sns.heatmap(cm, annot=True, fmt="d", cmap=pinky_cmap, xticklabels=class_names, yticklabels=class_names)
    plt.xlabel("Predicted")
    plt.ylabel("True")
    plt.title("Confusion Matrix for SoftMax")
    plt.show()

    
# Desenare matrice de confuzie
plot_confusion_matrix_softmax(predictions_df_softmax["true"], predictions_df_softmax["predicted"], class_names)


#AFISARE PROBABILITATI DE PREDICTIE


fig, ax = plt.subplots(3, 3, figsize=(20, 20))

for n in range(3):
    idx = predictions_df_relu512.patient_id.unique()[n]
    grid, mask, broken_patches, mask_proba = visualise_breast_tissue(idx, pred_df=predictions_df_relu512)

    ax[n, 0].imshow(grid, alpha=0.9)
    ax[n, 1].imshow(mask, alpha=0.8)
    ax[n, 1].imshow(grid, alpha=0.7)
    
    # Afișarea zonei cu cancer și probabilitatea asociată
    ax[n, 2].imshow(mask_proba[:, :, 0], cmap='Reds')  # Aici se schimba cmap pentru a reflecta schema de culori dorita

    for m in range(3):
        ax[n, m].set_xlabel("y-coord")
        ax[n, m].set_ylabel("x-coord")
        ax[n, m].grid(False)
        
    ax[n, 0].set_title("Breast tissue slice")
    ax[n, 1].set_title("Mask applied on \n cancer tissue")
    ax[n, 2].set_title("Cancer probability")


fig, ax = plt.subplots(3, 3, figsize=(20, 20))

for n in range(3):
    idx = predictions_df_relu1024.patient_id.unique()[n]
    grid, mask, broken_patches, mask_proba = visualise_breast_tissue(idx, pred_df=predictions_df_relu1024)

    ax[n, 0].imshow(grid, alpha=0.9)
    ax[n, 1].imshow(mask, alpha=0.8)
    ax[n, 1].imshow(grid, alpha=0.7)
    
    # Afișarea zonei cu cancer și probabilitatea asociată
    ax[n, 2].imshow(mask_proba[:, :, 0], cmap='Reds')  # Aici se schimba cmap pentru a reflecta schema de culori dorita

    for m in range(3):
        ax[n, m].set_xlabel("y-coord")
        ax[n, m].set_ylabel("x-coord")
        ax[n, m].grid(False)
        
    ax[n, 0].set_title("Breast tissue slice")
    ax[n, 1].set_title("Mask applied on \n cancer tissue")
    ax[n, 2].set_title("Cancer probability")


fig, ax = plt.subplots(3, 3, figsize=(20, 20))

for n in range(3):
    idx = predictions_df_softmax.patient_id.unique()[n]
    grid, mask, broken_patches, mask_proba = visualise_breast_tissue(idx, pred_df=predictions_df_softmax)

    ax[n, 0].imshow(grid, alpha=0.9)
    ax[n, 1].imshow(mask, alpha=0.8)
    ax[n, 1].imshow(grid, alpha=0.7)
    
    # Afișarea zonei cu cancer și probabilitatea asociată
    ax[n, 2].imshow(mask_proba[:, :, 0], cmap='Reds')  # Aici se schimba cmap pentru a reflecta schema de culori dorita

    for m in range(3):
        ax[n, m].set_xlabel("y-coord")
        ax[n, m].set_ylabel("x-coord")
        ax[n, m].grid(False)
        
    ax[n, 0].set_title("Breast tissue slice")
    ax[n, 1].set_title("Mask applied on \n cancer tissue")
    ax[n, 2].set_title("Cancer probability")


import matplotlib.pyplot as plt

# Lista cu ID-urile pacienților pe care dorești să-i vizualizezi
patient_ids = ['10260', '10273', '14210']  # Înlocuiește cu ID-urile reale

# Asigură-te că numărul de ID-uri pacienți nu depășește 3, deoarece avem 3 rânduri în subplots
if len(patient_ids) > 3:
    patient_ids = patient_ids[:3]

fig, ax = plt.subplots(len(patient_ids), 3, figsize=(20, 20))

for n, patient_id in enumerate(patient_ids):
    grid, mask, broken_patches, mask_proba = visualise_breast_tissue(patient_id, pred_df=predictions_df_relu512)

    ax[n, 0].imshow(grid, alpha=0.9)
    ax[n, 1].imshow(mask, alpha=0.8)
    ax[n, 1].imshow(grid, alpha=0.7)
    
    # Afișarea zonei cu cancer și probabilitatea asociată
    ax[n, 2].imshow(mask_proba[:, :, 0], cmap='Reds')  # Aici se schimbă cmap pentru a reflecta schema de culori dorită

    for m in range(3):
        ax[n, m].set_xlabel("y-coord")
        ax[n, m].set_ylabel("x-coord")
        ax[n, m].grid(False)
        
    ax[n, 0].set_title(f"Breast tissue slice of patient: {patient_id}")
    ax[n, 1].set_title(f"Cancer tissue colored red \n of patient: {patient_id}")
    ax[n, 2].set_title("Cancer probability")

plt.tight_layout()
plt.show()

	x	y	path
0	1001	1001	D:/input/breast-histopathology-images/IDC_regu...
1	1001	1051	D:/input/breast-histopathology-images/IDC_regu...
2	1001	1101	D:/input/breast-histopathology-images/IDC_regu...
3	1001	1151	D:/input/breast-histopathology-images/IDC_regu...
4	1001	1201	D:/input/breast-histopathology-images/IDC_regu...

	patient_id	path
0	10253	D:/input/breast-histopathology-images/IDC_regu...
1	10253	D:/input/breast-histopathology-images/IDC_regu...
2	10253	D:/input/breast-histopathology-images/IDC_regu...
3	10253	D:/input/breast-histopathology-images/IDC_regu...
4	10253	D:/input/breast-histopathology-images/IDC_regu...